package com.muki.spider.core;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.IDN;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.apache.log4j.Logger;

import com.muki.base.PoolExector;

/**
 * 背包爬虫
 * 爬虫原子类
 * 被观察者
 * @author ljh_2017
 *
 */
public class PackSpider {

	private Map<String,Object> properties = new HashMap<>();
	private final Logger log = Logger.getLogger(PackSpider.class);

	public void put(String key,Object value) {
		properties.put(key, value);
	}

	public Object getObject(String key) {
		return properties.getOrDefault(key,null);
	}

	public void put(String key,boolean value) {
		properties.put(key, value);
	}

	public boolean getBoolean(String key) {
		return (boolean)properties.getOrDefault(key,false);
	}

	public void put(String key,String value) {
		properties.put(key, value);
	}

	public String getString(String key) {
		return (String) properties.getOrDefault(key, null);
	}

	public void put(String key,int value) {
		properties.put(key, value);
	}

	public int getInt(String key) {
		return (int) properties.getOrDefault(key, 0);
	}

	public void put(String key,long value) {
		properties.put(key, value);
	}

	public long getLong(String key) {
		return (long)properties.getOrDefault(key, 0);
	}

	public void put(String key,double value) {
		properties.put(key, value);
	}

	public double getDouble(String key) {
		return (double) properties.getOrDefault(key, 0);
	}

	public void work() {
		String topic = getString("topic");
		if(topic==null) {
			return ;
		}
		UriQueueManagerPool pool = UriQueueManagerPool.getInstance();
		PoolExector.execute(new Runnable(){

			@Override
			public void run() {
				int count = 0;

				while(true) {
					try {
						String url = pool.getUrl(topic);
						if(url==null) {
							Thread.sleep(100);
							count++;
							log.info("爬取数据失败 "+count+" 次");
							if(count==100) {
								log.info(Thread.currentThread().getName()+" 爬虫关闭！");
								break;
							}
							continue;
						}

						int index = url.lastIndexOf(",");
						int depth = Integer.parseInt(url.substring(index+1));
						url = url.substring(0,index);
						log.info("正在抓取： { "+url+" }");
						crawling(url,topic,depth);
						Thread.sleep(100);
					} catch (InterruptedException | UnsupportedEncodingException e) {
						continue;
					}
				}
			}

		});
	}

	public void crawling(String uri,String topic,int depth) throws UnsupportedEncodingException {
		CloseableHttpClient client = HttpClients.createDefault();

		String regEx = "[\\u4e00-\\u9fa5]";
		Pattern p = Pattern.compile(regEx);
		Matcher m = p.matcher(uri);
		HttpGet method = null;
		while(m.find()) {
			String reg = m.group();
			uri=uri.replaceAll(reg, IDN.toASCII(reg));
		}

		if(uri==null)
			return;
		method = new HttpGet(uri);
		CloseableHttpResponse response;
		try {
			response = client.execute(method);
			HttpEntity entity = response.getEntity();
			String content = EntityUtils.toString(entity,"utf-8");
			//uri 原链接
			//result 爬去的原文
			//topic 主题
			//depth 深度
			ContentMessageParser.parse(uri,content, topic,depth);
		} catch (ClientProtocolException e) {
			return;
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

}
